﻿using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Text.RegularExpressions;
using System.Threading;

namespace ImprovedWebCrawler
{
    public class Crawler
    {
        private readonly string _startUrl;
        private readonly Uri _startUri;
        private readonly HashSet<string> _visitedUrls = new HashSet<string>();
        private readonly ConcurrentQueue<string> _urlsToCrawl = new ConcurrentQueue<string>();
        private readonly object _lock = new object();
        private bool _stopRequested = false;

        // 只处理这些扩展名的页面
        private readonly string[] _allowedExtensions = { ".htm", ".html", ".aspx", ".php", ".jsp", "/" };

        public event EventHandler<PageCrawledEventArgs> PageCrawled;
        public event EventHandler<PageErrorEventArgs> PageError;
        public event EventHandler CrawlCompleted;

        public Crawler(string startUrl)
        {
            _startUrl = startUrl;
            _startUri = new Uri(startUrl);
            _urlsToCrawl.Enqueue(startUrl);
        }

        public void Start()
        {
            _stopRequested = false;
            _visitedUrls.Clear();

            while (!_urlsToCrawl.IsEmpty && !_stopRequested)
            {
                if (_urlsToCrawl.TryDequeue(out string currentUrl))
                {
                    if (_visitedUrls.Contains(currentUrl))
                        continue;

                    try
                    {
                        CrawlPage(currentUrl);
                        _visitedUrls.Add(currentUrl);
                        PageCrawled?.Invoke(this, new PageCrawledEventArgs(currentUrl));
                    }
                    catch (Exception ex)
                    {
                        PageError?.Invoke(this, new PageErrorEventArgs(currentUrl, ex.Message));
                    }
                }
            }

            if (!_stopRequested)
            {
                CrawlCompleted?.Invoke(this, EventArgs.Empty);
            }
        }

        public void Stop()
        {
            _stopRequested = true;
        }

        private void CrawlPage(string url)
        {
            using (WebClient client = new WebClient())
            {
                client.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64)");
                string html = client.DownloadString(url);

                // 检查内容类型是否为HTML
                string contentType = client.ResponseHeaders["Content-Type"];
                if (contentType != null && !contentType.Contains("text/html"))
                {
                    return;
                }

                // 解析并处理页面中的链接
                ParseLinks(html, url);
            }
        }

        private void ParseLinks(string html, string baseUrl)
        {
            // 匹配所有href属性
            var linkRegex = new Regex(@"<a\s+(?:[^>]*?\s+)?href=([""'])(.*?)\1", RegexOptions.IgnoreCase);
            var matches = linkRegex.Matches(html);

            foreach (Match match in matches)
            {
                if (match.Groups.Count < 3) continue;

                string href = match.Groups[2].Value;

                // 跳过空链接和JavaScript链接
                if (string.IsNullOrEmpty(href) || href.StartsWith("javascript:"))
                    continue;

                try
                {
                    string absoluteUrl = GetAbsoluteUrl(baseUrl, href);

                    // 只处理与起始URL相同域名的链接
                    if (IsSameDomain(absoluteUrl))
                    {
                        // 只处理特定扩展名的URL
                        if (ShouldCrawlUrl(absoluteUrl))
                        {
                            if (!_visitedUrls.Contains(absoluteUrl))
                            {
                                _urlsToCrawl.Enqueue(absoluteUrl);
                            }
                        }
                    }
                }
                catch (UriFormatException)
                {
                    // 忽略格式不正确的URL
                }
            }
        }

        private string GetAbsoluteUrl(string baseUrl, string relativeUrl)
        {
            Uri baseUri = new Uri(baseUrl);

            // 处理各种形式的相对URL
            if (relativeUrl.StartsWith("//"))
            {
                return baseUri.Scheme + ":" + relativeUrl;
            }
            else if (relativeUrl.StartsWith("/"))
            {
                return new Uri(baseUri, relativeUrl).AbsoluteUri;
            }
            else if (relativeUrl.StartsWith("./"))
            {
                return new Uri(baseUri, relativeUrl.Substring(2)).AbsoluteUri;
            }
            else if (relativeUrl.StartsWith("../"))
            {
                return new Uri(baseUri, relativeUrl).AbsoluteUri;
            }
            else if (relativeUrl.StartsWith("http://") || relativeUrl.StartsWith("https://"))
            {
                return relativeUrl;
            }
            else
            {
                // 处理没有前导/的相对URL
                if (!baseUrl.EndsWith("/"))
                {
                    int lastSlash = baseUrl.LastIndexOf('/');
                    if (lastSlash > 8) // 8是"https://".Length
                    {
                        baseUrl = baseUrl.Substring(0, lastSlash + 1);
                    }
                }
                return new Uri(new Uri(baseUrl), relativeUrl).AbsoluteUri;
            }
        }

        private bool IsSameDomain(string url)
        {
            try
            {
                Uri uri = new Uri(url);
                return uri.Host.Equals(_startUri.Host, StringComparison.OrdinalIgnoreCase);
            }
            catch
            {
                return false;
            }
        }

        private bool ShouldCrawlUrl(string url)
        {
            // 检查URL是否以允许的扩展名结尾或是目录
            return _allowedExtensions.Any(ext =>
                url.EndsWith(ext, StringComparison.OrdinalIgnoreCase) ||
                !Path.HasExtension(url));
        }
    }

    public class PageCrawledEventArgs : EventArgs
    {
        public string Url { get; }

        public PageCrawledEventArgs(string url)
        {
            Url = url;
        }
    }

    public class PageErrorEventArgs : EventArgs
    {
        public string Url { get; }
        public string ErrorMessage { get; }

        public PageErrorEventArgs(string url, string errorMessage)
        {
            Url = url;
            ErrorMessage = errorMessage;
        }
    }
}